FROM jupyter/pyspark-notebook:spark-3.3.1
USER root

RUN echo "spark.jars.packages org.apache.spark:spark-sql-kafka-0-10_2.12:3.0.2,org.mongodb.spark:mongo-spark-connector:10.0.2" >> "${SPARK_HOME}/conf/spark-defaults.conf"
RUN echo "spark.eventLog.enabled true" >> "${SPARK_HOME}/conf/spark-defaults.conf"
RUN echo "spark.eventLog.dir /opt/spark/logs_event" >> "${SPARK_HOME}/conf/spark-defaults.conf"
RUN echo "spark.history.fs.logDirectory /opt/spark/logs_event" >> "${SPARK_HOME}/conf/spark-defaults.conf"

RUN mkdir -p /opt/spark/logs_event
